﻿using DotnetSpider.Core;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Zj.DotNetSpiderSample.Pipelines;

namespace Zj.DotNetSpiderSample
{
    class Program
    {
        static void Main(string[] args)
        {
            var startUrl = "https://www.cnblogs.com/";
            //var startUrls = new List<string>();
            //for (int i = 0; i < 200; i++)
            //{
            //    startUrls.Add($"{startUrl}#p{i+1}");
            //}
            var site = new Site()
            {
                CycleRetryTimes = 3,
                SleepTime = 3
            };
            var spider = Spider.Create(site,new CNBlogSpider())
                .AddPipeline(new CNBlogPipeline())
                //.AddStartUrls(startUrls)
                .AddStartUrl(startUrl);
            spider.ThreadNum = 16;
            spider.Run();

            Console.ReadKey();
        }
    }
}
